package org.domeos.framework.api.service.deployment.impl;

import io.fabric8.kubernetes.api.model.ContainerStatus;
import io.fabric8.kubernetes.api.model.Event;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodList;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.collections.CollectionUtils;
import org.domeos.basemodel.HttpResponseTemp;
import org.domeos.basemodel.ResultStat;
import org.domeos.exception.DataBaseContentException;
import org.domeos.exception.DeploymentEventException;
import org.domeos.exception.DeploymentTerminatedException;
import org.domeos.exception.K8sDriverException;
import org.domeos.framework.api.biz.OperationHistory;
import org.domeos.framework.api.biz.cluster.ClusterBiz;
import org.domeos.framework.api.biz.collection.CollectionBiz;
import org.domeos.framework.api.biz.configuration.ConfigurationBiz;
import org.domeos.framework.api.biz.deployment.DeployCollectionBiz;
import org.domeos.framework.api.biz.deployment.DeployEventBiz;
import org.domeos.framework.api.biz.deployment.DeploymentBiz;
import org.domeos.framework.api.biz.deployment.DeploymentStatusBiz;
import org.domeos.framework.api.biz.deployment.VersionBiz;
import org.domeos.framework.api.biz.event.K8SEventBiz;
import org.domeos.framework.api.biz.global.GlobalBiz;
import org.domeos.framework.api.biz.loadBalancer.LoadBalancerBiz;
import org.domeos.framework.api.biz.loadBalancer.LoadBalancerCollectionBiz;
import org.domeos.framework.api.biz.productline.ProductLineBiz;
import org.domeos.framework.api.biz.storage.StorageBiz;
import org.domeos.framework.api.consolemodel.deployment.ContainerConsole;
import org.domeos.framework.api.consolemodel.deployment.ContainerDraft;
import org.domeos.framework.api.consolemodel.deployment.DeploymentDetail;
import org.domeos.framework.api.consolemodel.deployment.DeploymentDraft;
import org.domeos.framework.api.consolemodel.deployment.DeploymentInfo;
import org.domeos.framework.api.consolemodel.deployment.EnvDraft;
import org.domeos.framework.api.consolemodel.deployment.LoadBalancerForDeploy;
import org.domeos.framework.api.consolemodel.deployment.VersionDraft;
import org.domeos.framework.api.consolemodel.deployment.VersionString;
import org.domeos.framework.api.consolemodel.deployment.VolumeMountConsole;
import org.domeos.framework.api.consolemodel.deployment.VolumeMountDraft;
import org.domeos.framework.api.consolemodel.event.EventInfo;
import org.domeos.framework.api.consolemodel.loadBalancer.KubeServiceDraft;
import org.domeos.framework.api.consolemodel.loadBalancer.NginxDraft;
import org.domeos.framework.api.controller.exception.ApiException;
import org.domeos.framework.api.controller.exception.PermitException;
import org.domeos.framework.api.mapper.domeos.global.GlobalMapper;
import org.domeos.framework.api.model.auth.User;
import org.domeos.framework.api.model.auth.related.Role;
import org.domeos.framework.api.model.cluster.Cluster;
import org.domeos.framework.api.model.cluster.related.ClusterWatcherDeployMap;
import org.domeos.framework.api.model.cluster.related.NamespaceInfo;
import org.domeos.framework.api.model.cluster.related.NodeInfo;
import org.domeos.framework.api.model.collection.CollectionAuthorityMap;
import org.domeos.framework.api.model.collection.CollectionResourceMap;
import org.domeos.framework.api.model.collection.related.ResourceType;
import org.domeos.framework.api.model.deployment.DeployCollection;
import org.domeos.framework.api.model.deployment.DeployEvent;
import org.domeos.framework.api.model.deployment.Deployment;
import org.domeos.framework.api.model.deployment.Policy;
import org.domeos.framework.api.model.deployment.UpdateJobResult;
import org.domeos.framework.api.model.deployment.Version;
import org.domeos.framework.api.model.deployment.related.Container;
import org.domeos.framework.api.model.deployment.related.CustomLabel;
import org.domeos.framework.api.model.deployment.related.DeployOperation;
import org.domeos.framework.api.model.deployment.related.DeployResourceStatus;
import org.domeos.framework.api.model.deployment.related.DeploymentAccessType;
import org.domeos.framework.api.model.deployment.related.DeploymentStatus;
import org.domeos.framework.api.model.deployment.related.DeploymentType;
import org.domeos.framework.api.model.deployment.related.Instance;
import org.domeos.framework.api.model.deployment.related.LabelSelector;
import org.domeos.framework.api.model.deployment.related.LoadBalancerForDeployDraft;
import org.domeos.framework.api.model.deployment.related.NetworkMode;
import org.domeos.framework.api.model.deployment.related.VersionType;
import org.domeos.framework.api.model.deployment.related.VolumePVC;
import org.domeos.framework.api.model.deployment.related.VolumeType;
import org.domeos.framework.api.model.global.GlobalInfo;
import org.domeos.framework.api.model.global.GlobalType;
import org.domeos.framework.api.model.global.Server;
import org.domeos.framework.api.model.loadBalancer.LoadBalancer;
import org.domeos.framework.api.model.loadBalancer.LoadBalancerCollection;
import org.domeos.framework.api.model.loadBalancer.related.DeployLoadBalancerMap;
import org.domeos.framework.api.model.loadBalancer.related.ForwardingRule;
import org.domeos.framework.api.model.loadBalancer.related.LoadBalancerCollectionType;
import org.domeos.framework.api.model.loadBalancer.related.LoadBalancerType;
import org.domeos.framework.api.model.operation.OperationRecord;
import org.domeos.framework.api.model.operation.OperationType;
import org.domeos.framework.api.model.productline.ProductLine;
import org.domeos.framework.api.model.storage.Storage;
import org.domeos.framework.api.model.storage.VolumeDeployMap;
import org.domeos.framework.api.service.auth.UserService;
import org.domeos.framework.api.service.deployment.DeploymentService;
import org.domeos.framework.api.service.deployment.DeploymentStatusManager;
import org.domeos.framework.api.service.event.EventService;
import org.domeos.framework.api.service.storage.StorageService;
import org.domeos.framework.engine.AuthUtil;
import org.domeos.framework.engine.ClusterRuntimeDriver;
import org.domeos.framework.engine.RuntimeDriver;
import org.domeos.framework.engine.coderepo.ReflectFactory;
import org.domeos.framework.engine.event.AutoDeploy.AutoUpdateInfo;
import org.domeos.framework.engine.exception.DaoException;
import org.domeos.framework.engine.exception.DriverException;
import org.domeos.framework.engine.k8s.LoadBalancerWrapper;
import org.domeos.framework.engine.k8s.NodeWrapper;
import org.domeos.framework.engine.k8s.handler.DeployResourceHandler;
import org.domeos.framework.engine.k8s.updater.EventChecker;
import org.domeos.framework.engine.k8s.util.Fabric8KubeUtils;
import org.domeos.framework.engine.k8s.util.KubeUtils;
import org.domeos.framework.engine.k8s.util.PodUtils;
import org.domeos.framework.engine.runtime.IResourceStatus;
import org.domeos.global.ClientConfigure;
import org.domeos.global.CurrentThreadInfo;
import org.domeos.global.GlobalConstant;
import org.domeos.util.CommonUtil;
import org.domeos.util.DateUtil;
import org.domeos.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

/**
 * 实例相关服务
 * @author haqiaolong
 *
 */
@Service
public class DeploymentServiceImpl implements DeploymentService {

    @Autowired
    DeploymentBiz deploymentBiz;
    
    @Autowired
    LoadBalancerBiz lbBiz;

    @Autowired
    CollectionBiz collectionBiz;
    
    @Autowired
    GlobalBiz globalBiz;
    
    @Autowired
    UserService userService; 

    @Autowired
    VersionBiz versionBiz;

    @Autowired
    LoadBalancerBiz loadBalancerBiz;

    @Autowired
    DeploymentStatusManager deploymentStatusManager;

    @Autowired
    IResourceStatus resourceStatus;

    @Autowired
    ClusterBiz clusterBiz;

    @Autowired
    DeployEventBiz deployEventBiz;

    @Autowired
    K8SEventBiz k8SEventBiz;

    @Autowired
    GlobalMapper globalMapper;

    @Autowired
    EventService eventService;

    @Autowired
    OperationHistory operationHistory;

    @Autowired
    DeploymentStatusBiz deploymentStatusBiz;

    @Autowired
    DeployCollectionBiz deployCollectionBiz;
    
    @Autowired
    ConfigurationBiz configurationBiz;
    
    @Autowired
    StorageBiz storageBiz;
    
    @Autowired
    StorageService storageService;
    
    @Autowired
    LoadBalancerCollectionBiz lbcBiz;

    @Autowired
    ProductLineBiz productLineBiz;

    
    private static Logger logger = LoggerFactory.getLogger(DeploymentServiceImpl.class);
    private static ExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    private final ResourceType resourceType = ResourceType.LOADBALANCER;
    private final ResourceType lbcresourceType = ResourceType.LOADBALANCER_COLLECTION;
    
    
    private void checkDeployPermit(int deployId, OperationType operationType) {
        int userId = CurrentThreadInfo.getUserId();
        AuthUtil.verify(userId, deployId, ResourceType.DEPLOY, operationType);
    }

    private void checkCreateDeployPermit(int collectionId, int clusterId) {
        int userId = CurrentThreadInfo.getUserId();
        AuthUtil.verify(userId, clusterId, ResourceType.CLUSTER, OperationType.MODIFY);
        AuthUtil.collectionVerify(userId, collectionId, ResourceType.DEPLOY_COLLECTION, OperationType.MODIFY, -1);

    }

    private void checkMigrateDeployPermit(int deployId, int collectionId) {
        int userId = CurrentThreadInfo.getUserId();
        AuthUtil.verify(userId, deployId, ResourceType.DEPLOY, OperationType.SET);
        AuthUtil.collectionVerify(userId, collectionId, ResourceType.DEPLOY_COLLECTION, OperationType.SET, -1);
    }
    
    public LoadBalancerCollection createlbc(int uid) {
    	List<LoadBalancerCollection> lbc = lbcBiz.getLoadBalancerCollection("inner_lb");
        LoadBalancerCollection lbcDraft;
        //当inner_lb lbc不存在，创建lbc
        if (lbc.size()==0) {
        	lbcDraft=new LoadBalancerCollection();
        	lbcDraft.setName("inner_lb");
        	lbcDraft.setCreatorId(uid);
        	lbcDraft.setType(LoadBalancerCollectionType.KUBE_PROXY);
        	lbcBiz.createLoadBalancerCollection(lbcDraft);
        	CollectionAuthorityMap collectionAuthorityMap = new CollectionAuthorityMap(lbcDraft.getId(),
        			lbcresourceType,
                    uid,
                    Role.MASTER,
                    System.currentTimeMillis()
                    );
        	collectionBiz.addAuthority(collectionAuthorityMap);
            
        }else {
        	lbcDraft = lbc.get(0);
        }
        return lbcDraft;
    }
    
    public String getunique(int deploycid,Deployment deployment) {
    	
    	DeployCollection deployc = deployCollectionBiz.getDeployCollectionByID(deploycid);
    	System.out.println(deploycid);
    	System.out.println(deployc);
    	String unique =  deployc.getName() + "-" + deployment.getName() + "-" + String.valueOf(deployc.getTreeId());
    	return unique;
    }
    
    
    public void addlabeltoversion(Version version,int deploycid) {
        //获取lbc  name,并通过treeid、deploylc、deploymentname拼凑唯一label
        Deployment deployment = deploymentBiz.getDeployment(version.getDeployId());
        //唯一label
        String unique = getunique(deploycid,deployment);
        //添加到version中
        CustomLabel cl = new CustomLabel();
        cl.setKey("unique");
        cl.setValue(unique);
        cl.setDescription("bind service");

        List<ContainerDraft> ctdraft = version.getContainerDrafts();
        if(CollectionUtils.isNotEmpty(version.getCustomLabels())) {
        	version.getCustomLabels().add(cl);
        }else {
        	List<CustomLabel> customLabels = new ArrayList<CustomLabel>();
        	customLabels.add(cl);
        	version.setCustomLabels(customLabels);
        }
    }
    


    @Override
    public int createDeployment(DeploymentDraft deploymentDraft) throws Exception {
        if (deploymentDraft == null) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_NOT_LEGAL, "deployment is null");
        }
        if (CurrentThreadInfo.getUser() == null) {
            throw new PermitException("no user logged in");
        }
        deploymentDraft.setCreatorId(CurrentThreadInfo.getUserId());
        checkCreateDeployPermit(deploymentDraft.getCollectionId(), deploymentDraft.getClusterId());
        DeployCollection deployCollection = deployCollectionBiz.getDeployCollection(deploymentDraft.getCollectionId());
        if (deployCollection == null) {
            throw ApiException.wrapMessage(ResultStat.DEPLOY_COLLECTION_NOT_EXIST,
                    "deploy collection " + deploymentDraft.getCollectionId() + " not exist");
        }

        Cluster cluster = clusterBiz.getClusterById(deploymentDraft.getClusterId());
        if (cluster == null) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_NOT_EXIST);
        }

        String deployName = deploymentDraft.getDeployName();
        List<Deployment> list = deploymentBiz.getListByName(GlobalConstant.DEPLOY_TABLE_NAME, deployName, Deployment.class);
        if (list != null && list.size() != 0) {
            for (Deployment one : list) {
                if (one.getClusterId() == deploymentDraft.getClusterId() &&
                        one.getNamespace().equals(deploymentDraft.getNamespace())) {
                    throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_EXIST);
                }
            }
        }
        //for innerService
//        LoadBalancerWrapper lbWrapper = new LoadBalancerWrapper().init(cluster.getId(), deploymentDraft.getNamespace());
//        if (lbWrapper.getLoadBalancerService(deployName) != null) {
//            throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_EXIST);
//        }
        deploymentDraft.setCreateTime(System.currentTimeMillis());
        Deployment deployment = deploymentDraft.toDeployment();
        deployment.setState(DeploymentStatus.STOP.name());
        deploymentBiz.createDeployment(deployment);
        
        //loadBalancer
        //获取lbc
        LoadBalancerCollection lbcDraft = createlbc(deploymentDraft.getUserId());
        LoadBalancer lb = null;
        LoadBalancerForDeployDraft loadBalancerDraft = deploymentDraft.getLoadBalancerDraft();
        if (deploymentDraft.getNetworkMode() != NetworkMode.HOST && loadBalancerDraft != null) {
            lb = loadBalancerDraft.toLoadBalancer(deployment);
            loadBalancerBiz.createLoadBalancer(lb);
            loadBalancerBiz.createLinkDeploy(new DeployLoadBalancerMap(deployment.getId(), lb.getId(), System.currentTimeMillis()));
            //lbc到lb映射
        	CollectionResourceMap resourceMap = new CollectionResourceMap(lb.getId(), 
        			deploymentDraft.getUserId(), 
                    resourceType, 
                    lbcDraft.getId(),
                    System.currentTimeMillis());
            collectionBiz.addResource(resourceMap);

        }
        
        Version version = deploymentDraft.toVersion();
        version.setDeployId(deployment.getId());
        version.setCreateTime(deployment.getCreateTime());
        
        //获取lbc  name,并通过treeid、deploylc、deploymentname拼凑唯一label
        addlabeltoversion(version,deployCollection.getId());
        
        //增加超分超卖配置
        double cpupercent = globalBiz.getCpuPercent();
        double memorypercent = globalBiz.getMemoryPercent();
        for(ContainerDraft cd:version.getContainerDrafts()) {
        	//页面只传了limits参数，首先更新requests为limits的值,然后再乘参数
        	cd.setCpuRequest(cd.getCpu());
        	cd.setMemRequest(cd.getMem());
        	cd.setCpuRequest(cd.getCpuRequest() * cpupercent);
        	cd.setMemRequest(cd.getMemRequest() * memorypercent);
        }
        
        
        try {
            versionBiz.insertVersionWithLogCollect(version, cluster);
        } catch (Exception e) {
            // failed
            deploymentBiz.removeById(GlobalConstant.DEPLOY_TABLE_NAME, deployment.getId());
            versionBiz.removeById(GlobalConstant.VERSION_TABLE_NAME, version.getId());
            if (lb != null) {
                loadBalancerBiz.removeLoadBalancer(lb.getId());
                loadBalancerBiz.removeLinkDeployByLoadBalancerId(lb.getId());
            }
            throw e;
        }
        
      
        
        // add volume_deploy_map
        for(ContainerConsole cc : deploymentDraft.getContainerConsoles()) {
        	for(VolumeMountConsole vmc: cc.getVolumeMountConsoles()) {
        		if ("PERSISTENTVOLUMECLAIM".equals(vmc.getVolumeType().name())) {
        			String storageName = vmc.getVolumePVC().getClaimName().split("-pvc")[0];
        			Storage storage = storageBiz.getStorageByName(storageName);
        			VolumeDeployMap volumeDeployMap = new VolumeDeployMap();
        			volumeDeployMap.setDeployId(deployment.getId());
        			volumeDeployMap.setVersionId(version.getId());
        			volumeDeployMap.setVolumeId(storage.getId());
        			volumeDeployMap.setCreateTime(System.currentTimeMillis());
        			storageBiz.addVolumeVersionMount(volumeDeployMap);
        		}
        	}
        }

        CollectionResourceMap resourceMap = new CollectionResourceMap(deployment.getId(),
                deploymentDraft.getCreatorId(),
                ResourceType.DEPLOY,
                deploymentDraft.getCollectionId(),
                System.currentTimeMillis());
        collectionBiz.addResource(resourceMap);
        
        //获取最新删除的deploy并绑定到之前的service
        
        //获取唯一label
        String unique = getunique(deploymentDraft.getCollectionId(), deployment);
        List<Deployment> olddeploys = deploymentBiz.getRemoveDeployment(cluster.getId(), deployName);
        //匹配新旧deploy
        Deployment olddeploy = null ;

        if (olddeploys.size()!=0) {
        	for(Deployment oldd:olddeploys) {
        		int b=0;
        		List<Version> versions = versionBiz.getAllVersionByDeployId(oldd.getId());
        		Version oldversion = new Version();
        		if (versions.size()!=0) {
        			oldversion=versions.get(0);
        		}
        		for (CustomLabel ctl:version.getCustomLabels()) {
        			if (ctl.getKey().equals("unique")) {
        				if (ctl.getValue().equals(unique)) {
        					olddeploy = oldd;
        					b=1;
        					break;
        				}
        			}
        		}
        		if (b==1) {
        			break;
        		}
        	}	
        }
        //开始绑定到svc
        if (olddeploy != null) {
        	//更新lb表
        	LoadBalancer oldlb = loadBalancerBiz.getInnerLoadBalancerByDeployId(olddeploy.getId());
        	if (oldlb != null) {
        		KubeServiceDraft kbsd = oldlb.getServiceDraft();
            	if (kbsd != null) {
            		kbsd.setDeployId(deployment.getId());
            		kbsd.setDeployName(deployment.getName());
            		loadBalancerBiz.updateLoadBalancer(oldlb);
            		//更新lb_deploy_map表
                	DeployLoadBalancerMap dlbmap = new DeployLoadBalancerMap();
                	dlbmap.setLoadBalancerId(oldlb.getId());
                	dlbmap.setDeployId(deployment.getId());
                	loadBalancerBiz.updateLinkDeploy(dlbmap);
                	//更新实际svc
                	LoadBalancerWrapper wrapper = new LoadBalancerWrapper().init(deployment.getClusterId(), deployment.getNamespace());
                	wrapper.updateLoadBalanceService(oldlb);
            	}
        	}
        }
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                resourceMap.getResourceId(),
                resourceMap.getResourceType(),
                OperationType.SET,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));

        // TODO(sparkchen)
        logger.info("create deploy succeed, deployId={}, creatorId={}, collectionId={}",
                deployment.getId(), deploymentDraft.getCreatorId(), deploymentDraft.getCollectionId());

        return deployment.getId();
    }

    private List<CollectionAuthorityMap> getCollectionList() {
        int userId = CurrentThreadInfo.getUserId();
        return AuthUtil.getCollectionList(userId, ResourceType.DEPLOY_COLLECTION);
    }

    @Override
    public void removeDeployment(int deployId) throws IOException {
        checkDeployPermit(deployId, OperationType.DELETE);
        Deployment deployment = deploymentBiz.getById(GlobalConstant.DEPLOY_TABLE_NAME, deployId, Deployment.class);
        DeploymentStatus deploymentStatus = deploymentStatusBiz.getDeploymentStatus(deployId);
        if (deploymentStatus == null || !deploymentStatus.equals(DeploymentStatus.STOP)) {
            throw ApiException.wrapMessage(ResultStat.CANNOT_DELETE_DEPLOYMENT, "deployment status is " + deployment.getState() + " now");
        }
        //loadBalancerBiz.removeLoadBalancerByDeployId(deployId);
        versionBiz.disableAllVersion(deployId);
        deploymentBiz.removeById(GlobalConstant.DEPLOY_TABLE_NAME, deployId);
        eventService.deleteDeploymentEvent(deployment.getClusterId(), deployment);
        
        configurationBiz.removeConfigurationVersionMapByDeployId(deployId);
        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
            clusterBiz.deleteClusterWatchDeployMapByDeployId(deployId);
        } else {
            collectionBiz.deleteResourceByResourceIdAndResourceType(deployId, ResourceType.DEPLOY);
        }
        
        //delete from the volume_deploy_map
        storageBiz.removeVolumeVersionMapByDeployId(deployId);
        
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.DELETE,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
    }

    @Override
    public void modifyDeployment(int deployId, DeploymentDraft deploymentDraft) throws Exception {
        checkDeployPermit(deployId, OperationType.MODIFY);
        if (deploymentDraft == null) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_NOT_LEGAL, "deployment is null");
        }

        String errInfo = deploymentDraft.checkLegality();
        if (!StringUtils.isBlank(errInfo)) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_NOT_LEGAL, errInfo);
        }
        Deployment oldDeploy = deploymentBiz.getDeployment(deployId);
        if (oldDeploy == null) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_NOT_EXIST);
        }
        Deployment newDeploy = deploymentDraft.toDeployment();

        newDeploy.setId(oldDeploy.getId());
        newDeploy.setState(oldDeploy.getState());
        //for now, we just allow to update the description of the deployment
        oldDeploy.setDescription(newDeploy.getDescription());
        deploymentBiz.update(oldDeploy);
        // TODO(sparkchen)
        // Acturally Operation

        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.MODIFY,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));

    }
    
    @Override
    public void modifyDeploymentDescription(int deployId, String description) {
        checkDeployPermit(deployId, OperationType.MODIFY);

        Deployment oldDeploy = deploymentBiz.getDeployment(deployId);
        if (oldDeploy == null) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_NOT_EXIST);
        }

        deploymentBiz.updateDescription(deployId, description);

        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.MODIFY,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
    }

    @Override
    public List<DeploymentInfo> listDeployment() throws IOException {
        List<CollectionAuthorityMap> authorityMaps = getCollectionList();
        Map<Integer, Boolean> deletableMap = new HashMap<>();
        int userId = CurrentThreadInfo.getUserId();
        boolean isAdmin = AuthUtil.isAdmin(userId);
        for (CollectionAuthorityMap authorityMap : authorityMaps) {
            if (isAdmin || authorityMap.getRole() == Role.MASTER) {
                deletableMap.put(authorityMap.getCollectionId(), true);
            } else {
                deletableMap.put(authorityMap.getCollectionId(), false);
            }
        }

        List<CollectionResourceMap> resources = collectionBiz.getResourcesByAuthorityMaps(ResourceType.DEPLOY, authorityMaps);
        return listDeployment(resources, deletableMap);
    }

    @Override
    public List<DeploymentInfo> listDeployment(int collectionId) throws IOException {

        int userId = CurrentThreadInfo.getUserId();
        boolean deletable;
        try {
            AuthUtil.collectionVerify(userId, collectionId, ResourceType.DEPLOY_COLLECTION, OperationType.DELETE, -1);
            deletable = true;
        } catch (Exception ignore) {
            deletable = false;
        }
        Map<Integer, Boolean> deletableMap = new HashMap<>();
        deletableMap.put(collectionId, deletable);

        List<CollectionResourceMap> resources = collectionBiz.getResourcesByCollectionIdAndResourceType(collectionId, ResourceType.DEPLOY);
        return listDeployment(resources, deletableMap);

    }

    private List<DeploymentInfo> listDeployment(List<CollectionResourceMap> resources, Map<Integer, Boolean> deletableMap) throws IOException {
        if (resources == null || resources.size() == 0) {
            return new ArrayList<>(1);
        }
        int userId = AuthUtil.getUserId();
        List<GetDeploymentInfoTask> deploymentInfoTasks = new LinkedList<>();
        for (CollectionResourceMap resourceMap : resources) {
            deploymentInfoTasks.add(new GetDeploymentInfoTask(resourceMap, deletableMap, userId));
        }
        List<DeploymentInfo> deploymentInfos = ClientConfigure.executeCompletionService(deploymentInfoTasks);
        // sort by createTime
        Collections.sort(deploymentInfos, new Comparator<DeploymentInfo>() {
            @Override
            public int compare(DeploymentInfo o1, DeploymentInfo o2) {
                return ((Long) o2.getCreateTime()).compareTo(o1.getCreateTime());
            }
        });
        return deploymentInfos;
    }


    private class GetDeploymentInfoTask implements Callable<DeploymentInfo> {
        CollectionResourceMap collectionResourceMap;
        Map<Integer, Boolean> deletableMap;
        int userId;

        private GetDeploymentInfoTask(CollectionResourceMap collectionResourceMap, Map<Integer, Boolean> deletableMap, int userId) {
            this.collectionResourceMap = collectionResourceMap;
            this.deletableMap = deletableMap;
            this.userId = userId;
        }

        @Override
        public DeploymentInfo call() throws Exception {
            Deployment deployment = deploymentBiz.getDeployment(collectionResourceMap.getResourceId());
            if (deployment == null) {
                return null;
            }
            DeploymentInfo deploymentInfo = new DeploymentInfo(deployment);
            deploymentInfo.setDeploymentStatus(deploymentStatusBiz.getDeploymentStatus(deployment.getId()));
            DeployResourceStatus deployResourceStatus = resourceStatus.getDeployResourceStatusById(deployment.getId());
            boolean deletable = deletableMap.get(collectionResourceMap.getCollectionId()) || userId == collectionResourceMap.getCreatorId();
            deploymentInfo.setDeletable(deletable);
            if (deployResourceStatus != null) {
                deploymentInfo.setCpuTotal(deployResourceStatus.getCpuTotal());
                deploymentInfo.setCpuUsed(deployResourceStatus.getCpuUsed());
                deploymentInfo.setMemoryTotal(deployResourceStatus.getMemTotal());
                deploymentInfo.setMemoryUsed(deployResourceStatus.getMemUsed());
            }
            // set cluster name
            Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
            if (cluster == null) {
                throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST,
                        "deployment " + deployment.getName() + " requires the cluster (clusterId: " + deployment.getClusterId() + ") information");
            }
            deploymentInfo.setClusterName(cluster.getName());
            String serviceDnsName = "";
            List<LoadBalancer> loadBalancers = loadBalancerBiz.getLoadBalancersByDeploy(deployment.getId());
            if (loadBalancers != null) {
                for (LoadBalancer lb : loadBalancers) {
                    if (lb.getType() != LoadBalancerType.NGINX) {
                      serviceDnsName += CommonUtil.generateServiceDnsName(
                              lb.getNamespace(), cluster.getDomain(), lb.getName()) + " ";
                    }
                }
            }
            VersionType versionType = deployment.getVersionType() == null ? VersionType.CUSTOM : deployment.getVersionType();
            deploymentInfo.setVersionType(versionType);
            deploymentInfo.setServiceDnsName(serviceDnsName);
            deploymentInfo.setCreateTime(deployment.getCreateTime());
            deploymentInfo.setDeployTypeShow(deployment.getDeploymentType().getShowName());
            deploymentInfo.setProductLine(getProductLine(deployment));
            return deploymentInfo;
        }
    }
    
    private String getProductLine(Deployment deployment) throws DaoException {
    	ProductLine productLine;
		try {
			CollectionResourceMap collectionResourceMap = collectionBiz.getResourceByResourceIdAndResourceType(deployment.getId(), ResourceType.DEPLOY);
			DeployCollection deployCollection = deployCollectionBiz.getDeployCollectionByID(collectionResourceMap.getCollectionId());
			productLine = productLineBiz.selectById(deployCollection.getTreeId());
		} catch (Exception e) {
			e.printStackTrace();
			return null;
		}
		return getFullProductLineName(productLine, new String()); 
    }
    private String MergeProductLineName(ProductLine productLine) {
    	String productLineName;
    	if(productLine.getDescription().isEmpty()) {
    		productLineName = productLine.getName();
    	}else {
    		productLineName = productLine.getDescription();
    	}
    	return productLineName + "/";
    }
    private String getFullProductLineName(ProductLine productLine, String productLineName) throws DaoException {
    	if(productLine.getLevel() <= 2) {
    		return MergeProductLineName(productLine) + productLineName;
    	}
    	ProductLine parentProductLine = productLineBiz.selectById(productLine.getpId());
    	productLineName = MergeProductLineName(productLine) + productLineName;
    	return getFullProductLineName(parentProductLine, productLineName);
    }

    @Override
    public DeploymentDetail getDeployment(int deployId) throws DeploymentEventException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.GET);

        Deployment deployment = deploymentBiz.getDeployment(deployId);

        if (deployment == null) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_NOT_EXIST);
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.SERVER_INTERNAL_ERROR, "ClusterId: " + deployment.getClusterId() + " not exists");
        }

        DeploymentDetail deploymentDetail = new DeploymentDetail();
        deploymentDetail.setDeployId(deployment.getId());
        deploymentDetail.setDeployName(deployment.getName());
        deploymentDetail.setNamespace(deployment.getNamespace());
        deploymentDetail.setScalable(deployment.isScalable());
        deploymentDetail.setDefaultReplicas(deployment.getDefaultReplicas());
        deploymentDetail.setClusterName(cluster.getName());
        deploymentDetail.setClusterId(cluster.getId());
        deploymentDetail.setClusterLog(cluster.getClusterLog());
        deploymentDetail.setNetworkMode(deployment.getNetworkMode());
        deploymentDetail.setLastUpdateTime(deployment.getLastUpdateTime());
        deploymentDetail.setHealthChecker(deployment.getHealthChecker());
        VersionType versionType = deployment.getVersionType() == null ? VersionType.CUSTOM : deployment.getVersionType();
        deploymentDetail.setVersionType(versionType);

        // set deploymentStatus
        deploymentDetail.setDeploymentStatus(deploymentStatusBiz.getDeploymentStatus(deployId));

        deploymentDetail.setExposePortNum(deployment.getExposePortNum());
        deploymentDetail.setAccessType(DeploymentAccessType.DIY);
        LoadBalancerWrapper wrapper = null;
        try {
        	wrapper = new LoadBalancerWrapper().init(cluster.getId(), deployment.getNamespace());
        }catch(Exception e) {
        	//nothing to do
        }
        //loadBalancer
        if (deployment.getNetworkMode() != NetworkMode.HOST) {
            final List<LoadBalancer> loadBalancers = loadBalancerBiz.getLoadBalancersByDeploy(deployId);
            if (loadBalancers != null && loadBalancers.size() != 0) {
                List<LoadBalancerForDeploy> lbForDeploys = new ArrayList<LoadBalancerForDeploy>();
                for (LoadBalancer lb : loadBalancers) {
                    if (lb.getType() == LoadBalancerType.NGINX) {
                        NginxDraft nginxDraft = lb.getNginxDraft();
                        if (nginxDraft != null && nginxDraft.getRules() != null) {
                            List<ForwardingRule> rules = nginxDraft.getRules();
                            List<ForwardingRule> rulesnew = new ArrayList<ForwardingRule>();
                            for (ForwardingRule rule : rules) {
                                if (rule.getDeployId() == deployId) {
                                    rulesnew.add(rule);
                                }
                            }
                            nginxDraft.setRules(rulesnew);
                        }
                        LoadBalancerForDeploy lbForDeploy = new LoadBalancerForDeploy(lb);
                        CollectionResourceMap resourceMap = collectionBiz.getResourceByResourceIdAndResourceType(lb.getId(), ResourceType.LOADBALANCER);
                        if (resourceMap != null) {
                            lbForDeploy.setLbcId(resourceMap.getCollectionId());
                        }
                        lbForDeploys.add(lbForDeploy);
                    } else if (lb.getType() == LoadBalancerType.INNER_SERVICE) {
                        String dnsName = CommonUtil.generateServiceDnsName(lb.getNamespace(), cluster.getDomain(), lb.getName()) + " ";
                        LoadBalancerForDeploy lbForDeploy = new LoadBalancerForDeploy(lb);
                        lbForDeploy.setDnsName(dnsName);
                        String ClusterIP = null;
                        try {
                        	ClusterIP = wrapper.getLoadBalancerService(lb).getSpec().getClusterIP();
                        }catch(Exception e) {
                        	//nothing to do
                        }
                        lbForDeploy.setClusterIP(ClusterIP);
                        lbForDeploys.add(lbForDeploy);
                    } else {
                        String dnsName = CommonUtil.generateServiceDnsName(lb.getNamespace(), cluster.getDomain(), lb.getName()) + " ";
                        LoadBalancerForDeploy lbForDeploy = new LoadBalancerForDeploy(lb);
                        lbForDeploy.setDnsName(dnsName);
                        CollectionResourceMap resourceMap = collectionBiz.getResourceByResourceIdAndResourceType(lb.getId(), ResourceType.LOADBALANCER);
                        if (resourceMap != null) {
                            lbForDeploy.setLbcId(resourceMap.getCollectionId());
                        }
                        lbForDeploys.add(lbForDeploy);
                    }
                }
                // set loadBalancer
                Collections.sort(lbForDeploys, new LoadBalancerForDeploy.LoadBalancerForDeployComparator());
                deploymentDetail.setLbForDeploys(lbForDeploys);
            }
        }
        // set deployment status
        deploymentDetail.setDeploymentStatus(deploymentStatusBiz.getDeploymentStatus(deployId));

        // set current replicas
        long currentReplicas;
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, " There is no RuntimeDriver for cluster(" + cluster.toString() + ").");
        }
        currentReplicas = driver.getTotalReplicasByDeployment(deployment);
        deploymentDetail.setCurrentReplicas(currentReplicas);

        // get current versions
        List<Version> versions = driver.getCurrnetVersionsByDeployment(deployment);
        if (versions == null || versions.isEmpty()) {
            deploymentDetail.setCurrentVersions(null);
        } else {
            List<VersionDraft> versionDrafts = new ArrayList<>(versions.size());
            for (Version version : versions) {
                VersionDraft versionDraft = new VersionDraft(version, deployment, cluster);
                DeployResourceHandler deployResourceHandler = ReflectFactory.createDeployResourceHandler(
                        deployment.getDeploymentType().getDeployClassName(), null, deployment);
                VersionString versionString = null;
                if (deployResourceHandler != null) {
                    List<LoadBalancer> lbs = loadBalancerBiz.getInnerAndExternalLoadBalancerByDeployId(deployId);
                    versionString = deployResourceHandler.getVersionString(version, lbs, buildExtraEnv(cluster));
                }
                if (versionString != null) {
                    versionString.setPodSpecStr(version.getPodSpecStr());
                }
                versionDraft.setVersionString(versionString);
                versionDrafts.add(versionDraft);
            }
            deploymentDetail.setCurrentVersions(versionDrafts);
        }
        deploymentDetail.setHealthChecker(deployment.getHealthChecker());
        if (deployment.getDeploymentType() == null) {
            deploymentDetail.setDeploymentType(DeploymentType.REPLICATIONCONTROLLER);
        } else {
            deploymentDetail.setDeploymentType(deployment.getDeploymentType());
        }

        boolean deletable;
        try {
            AuthUtil.verify(CurrentThreadInfo.getUserId(), deployId, ResourceType.DEPLOY, OperationType.DELETE);
            deletable = true;
        } catch (Exception ignore) {
            deletable = false;
        }
        deploymentDetail.setDeployTypeShow(deployment.getDeploymentType().getShowName());
        deploymentDetail.setDeletable(deletable);
        deploymentDetail.setDescription(deployment.getDescription());

        return deploymentDetail;
    }

    @Override
    public void migrateDeployment(int deployId, int collectionId) throws IOException, DeploymentEventException {
        checkMigrateDeployPermit(deployId, collectionId);
        CollectionResourceMap oldDeployResource = collectionBiz.getResourceByResourceIdAndResourceType(deployId, ResourceType.DEPLOY);
        if (oldDeployResource == null) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOY_COLLECTION_NOT_EXIST);
        }
        if (oldDeployResource.getCollectionId() == collectionId) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOY_IN_DEPLOY_COLLECTION);
        }
        oldDeployResource.setCollectionId(collectionId);
        oldDeployResource.setUpdateTime(System.currentTimeMillis());
        collectionBiz.modifyCollectionResourceMap(oldDeployResource);
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.MODIFY,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));

    }

    @Override
    public void stopDeployment(int deployId)
            throws IOException, DeploymentEventException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        if (deployment.getDeploymentType() != DeploymentType.DAEMONSET) {
            deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.STOPPING);
        } else {
            DeployEvent deployEvent = deployEventBiz.getNewestEventByDeployId(deployId);
            if (!deployEvent.eventTerminated() && deployEvent.getOperation() != DeployOperation.STOP) {
                deploymentStatusManager.failedEventForDeployment(deployId, null, "Fail the current event");
            } else if (deployEvent.getOperation() == DeployOperation.STOP) {
                throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_STOP_FAILED, "You are already in stop status");
            }
        }

        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }

        try {
            deploymentBiz.updateState(GlobalConstant.DEPLOY_TABLE_NAME, DeploymentStatus.STOPPING.name(), deployId);
            driver.stopDeploy(deployment, CurrentThreadInfo.getUser());
            // add operation record
            operationHistory.insertRecord(new OperationRecord(
                    deployId,
                    ResourceType.DEPLOY,
                    OperationType.STOP,
                    CurrentThreadInfo.getUserId(),
                    CurrentThreadInfo.getUserName(),
                    "OK",
                    "",
                    System.currentTimeMillis()
            ));
        } catch (DeploymentEventException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_STOP_FAILED, e.getMessage());
        }
    }

    @Override
    public void startDeployment(int deployId, int versionId, int replicas)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }

        Version version = versionBiz.getVersion(deployId, versionId);
        if (version == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such version:" + versionId);
        }
        if (version.isDeprecate()) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_START_FAILED, "can't start deprecated version");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.DEPLOYING);
        if (deployment.getDeploymentType() == DeploymentType.DAEMONSET) {
            try {
                deployment.setDefaultReplicas(getNodeCounts(deployment, version.getLabelSelectors()));
            } catch (Exception e) {
                throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "error with node label selectors: " + version.getLabelSelectors());
            }
        }

        if (deployment.getDeploymentType() == DeploymentType.DAEMONSET) {
            replicas = getNodeCounts(deployment, version.getLabelSelectors());
        }
        deployment.setState(DeploymentStatus.DEPLOYING.name());
        deployment.setLastUpdateTime(System.currentTimeMillis());
        if (replicas > 0) {
            deployment.setDefaultReplicas(replicas);
        }

        deploymentBiz.update(deployment);
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }

        try {
            List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
            driver.startDeploy(deployment, version, CurrentThreadInfo.getUser(), allExtraEnvs);
            // add operation record
            operationHistory.insertRecord(new OperationRecord(
                    deployId,
                    ResourceType.DEPLOY,
                    OperationType.START,
                    CurrentThreadInfo.getUserId(),
                    CurrentThreadInfo.getUserName(),
                    "OK",
                    "",
                    System.currentTimeMillis()
            ));
        } catch (DriverException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_START_FAILED, e.getMessage());
        }
    }

    @Override
    public HttpResponseTemp<?> startUpdate(int deployId, int versionId, int replicas, Policy policy)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        AutoUpdateInfo autoUpdateInfo = new AutoUpdateInfo();
        autoUpdateInfo.setDeployId(deployId);
        autoUpdateInfo.setVersionId(versionId);
        autoUpdateInfo.setReplicas(replicas);
        autoUpdateInfo.setPolicy(policy);

        startUpdate(autoUpdateInfo, true);
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.UPDATE,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap(null);
    }

    public void startUpdate(AutoUpdateInfo autoUpdateInfo, boolean check)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        // ** get deployment and version
        int deployId = autoUpdateInfo.getDeployId();
        int versionId = autoUpdateInfo.getVersionId();
        int replicas = autoUpdateInfo.getReplicas();
        Policy policy = autoUpdateInfo.getPolicy();
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }
        User user;
        if (check) {
            checkDeployPermit(deployId, OperationType.MODIFY);
            user = CurrentThreadInfo.getUser();
        } else {
            user = new User();
            user.setId(-1);
            user.setUsername("GDocker");
        }

        Version version = versionBiz.getVersion(deployId, versionId);
        if (version == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such version:" + versionId);
        }
        if (version.isDeprecate()) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_UPDATE_FAILED, "can't update deprecated version");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.UPDATING);
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }
        if (deployment.getDeploymentType() == DeploymentType.DAEMONSET) {
            try {
                deployment.setDefaultReplicas(getNodeCounts(deployment, version.getLabelSelectors()));
            } catch (Exception e) {
                throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "error with node label selectors: " + version.getLabelSelectors());
            }
        }
        try {
            List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
            if (replicas > 0) {
                deployment.setDefaultReplicas(replicas);
            }
            driver.startUpdate(deployment, versionId, allExtraEnvs, user, policy);
        } catch (DeploymentEventException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_UPDATE_FAILED, e.getMessage());
        }

        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.UPDATING.name());
        deploymentBiz.update(deployment);
    }
    
    public HttpResponseTemp<?> sreStartUpdate(AutoUpdateInfo autoUpdateInfo, boolean check , User user)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        int deployId = autoUpdateInfo.getDeployId();
        int versionId = autoUpdateInfo.getVersionId();
        int replicas = autoUpdateInfo.getReplicas();
        Policy policy = autoUpdateInfo.getPolicy();
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deployment:" + deployId);
        }
        Version version = versionBiz.getVersion(deployId, versionId);
        if (version == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such version:" + versionId);
        }
        if (version.isDeprecate()) {
        	return ResultStat.PARAM_ERROR.wrap("can't update deprecated version");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready");
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.UPDATING);
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist");
        }
        
        try {
            List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
            if (replicas > 0) {
                deployment.setDefaultReplicas(replicas);
            }
            driver.startUpdate(deployment, versionId, allExtraEnvs, user, policy);
        } catch (DeploymentEventException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            logger.error("deployment update failed!",e.getMessage());
            return ResultStat.PARAM_ERROR.wrap("deployment update failed!");
        }

        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.UPDATING.name());
        deploymentBiz.update(deployment);
        
        return ResultStat.OK.wrap();
    }

    @Override
    public void updateJobResult(UpdateJobResult updateJobResult) {
        if (updateJobResult == null) {
            return;
        }
        try {
            DeployEvent deployEvent = deployEventBiz.getEvent(Long.valueOf(updateJobResult.getEventId()));
            Deployment deployment = deploymentBiz.getDeployment(Integer.valueOf(updateJobResult.getDeployId()));
            if (deployEvent == null || deployment == null) {
                return;
            }
            if (updateJobResult.getResultCode() > 200) {
                deploymentStatusManager.failedEvent(Long.valueOf(updateJobResult.getEventId()), deployEvent.getCurrentSnapshot(),
                        updateJobResult.getResultMsg());
            }
            EventChecker eventChecker = new EventChecker(deployment, deployEvent);
            eventChecker.checkEvent();
        } catch (IOException | DeploymentEventException | DeploymentTerminatedException | DataBaseContentException e) {
            logger.warn("meet error in update job " + updateJobResult.getJobName() + ", deployId=" + updateJobResult.getDeployId()
                    + ", eventId=" + updateJobResult.getEventId());
        }
    }

    @Override
    public HttpResponseTemp<?> startRollback(int deployId, int versionId, int replicas, Policy policy)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.MODIFY);
        // ** get deployment and version
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }
        Version version = versionBiz.getVersion(deployId, versionId);
        if (version == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such version:" + versionId);
        }
        if (version.isDeprecate()) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_UPDATE_FAILED, "can't rollback deprecated version");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.BACKROLLING);

        // check status
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }
        if (deployment.getDeploymentType() == DeploymentType.DAEMONSET) {
            try {
                deployment.setDefaultReplicas(getNodeCounts(deployment, version.getLabelSelectors()));
            } catch (Exception e) {
                throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "error with node label selectors: " + version.getLabelSelectors());
            }
        }
        try {
            List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
            if (replicas > 0) {
                deployment.setDefaultReplicas(replicas);
            }
            driver.rollbackDeploy(deployment, versionId, allExtraEnvs, CurrentThreadInfo.getUser(), policy);
        } catch (DeploymentEventException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            ResultStat.DEPLOYMENT_UPDATE_FAILED.wrap(e.getMessage());
        }

        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.BACKROLLING.name());
        deploymentBiz.update(deployment);
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.ROLLBACK,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap(null);
    }

    @Override
    public HttpResponseTemp<?> scaleUpDeployment(int deployId, int versionId, int replicas)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "watcher cannot be scale");
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.UPSCALING);
        if (deployment.getDeploymentType() == DeploymentType.DAEMONSET) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "Daemon Set can not scale up");
        }
        deployment.setDefaultReplicas(replicas);
        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.UPSCALING.name());
        deploymentBiz.update(deployment);

        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }
        List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
        driver.scaleUpDeployment(deployment, versionId, replicas, allExtraEnvs, CurrentThreadInfo.getUser());
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.SCALEUP,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap(null);
    }

    @Override
    public HttpResponseTemp<?> scaleDownDeployment(int deployId, int versionId, int replicas)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {

        checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);

        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "watcher cannot be scale");
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.DOWNSCALING);

        if (deployment.getDeploymentType() == DeploymentType.DAEMONSET) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "Daemon Set can not scale up");
        }
        deployment.setDefaultReplicas(replicas);
        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.DOWNSCALING.name());
        deploymentBiz.update(deployment);


        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }
        List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
        driver.scaleDownDeployment(deployment, versionId, replicas, allExtraEnvs, CurrentThreadInfo.getUser());
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.SCALEDOWN,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap(null);
    }

    @Override
    public HttpResponseTemp<?> scaleDaemonSet(int deployId, int versionId, List<LabelSelector> labelSelectors)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);

        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "watcher cannot be scale");
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        Version version = versionBiz.getVersion(deployId, versionId);

        if (version == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such version: " + versionId + "in deployment:" + deployId);
        }
        DeploymentStatus deploymentStatus;
        int checkresult = compareLabelSelectors(deployment, version.getLabelSelectors(), labelSelectors);
        if (checkresult < 0) {
            deploymentStatus = DeploymentStatus.UPSCALING;
        } else if (checkresult > 0) {
            deploymentStatus = DeploymentStatus.DOWNSCALING;
        } else {
            return ResultStat.OK.wrap(null);
        }
        int replicas = getNodeCounts(deployment, labelSelectors);

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), deploymentStatus);

        if (deployment.getDeploymentType() != DeploymentType.DAEMONSET) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "deployment can not scale use labelselectors");
        }
        deployment.setDefaultReplicas(replicas);
        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(deploymentStatus.name());
        deploymentBiz.update(deployment);


        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, "cluster: " + cluster.toString());
        }
        List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
        version.setLabelSelectors(labelSelectors);
        versionBiz.updateLabelSelector(version);
        if (deploymentStatus == DeploymentStatus.DOWNSCALING) {
            driver.scaleDownDeployment(deployment, versionId, replicas, allExtraEnvs, CurrentThreadInfo.getUser());
            // add operation record
            operationHistory.insertRecord(new OperationRecord(
                    deployId,
                    ResourceType.DEPLOY,
                    OperationType.SCALEDOWN,
                    CurrentThreadInfo.getUserId(),
                    CurrentThreadInfo.getUserName(),
                    "OK",
                    "",
                    System.currentTimeMillis()
            ));
        } else {
            driver.scaleUpDeployment(deployment, versionId, replicas, allExtraEnvs, CurrentThreadInfo.getUser());
            // add operation record
            operationHistory.insertRecord(new OperationRecord(
                    deployId,
                    ResourceType.DEPLOY,
                    OperationType.SCALEUP,
                    CurrentThreadInfo.getUserId(),
                    CurrentThreadInfo.getUserName(),
                    "OK",
                    "",
                    System.currentTimeMillis()
            ));
        }

        return ResultStat.OK.wrap(null);
    }
    
    
    @Override
    public HttpResponseTemp<?> sreListDeployEvent(int deployId) throws IOException {
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deploy in database");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such cluster in database, clusterId = \" + deployment.getClusterId()");
        }
        if (deployId < 1) {
        	return ResultStat.PARAM_ERROR.wrap("deployId is illegal");
        }
        List<DeployEvent> deployEvents = deployEventBiz.getEventByDeployId(deployId);
        if(deployEvents == null || deployEvents.size() < 0){
        	return ResultStat.PARAM_ERROR.wrap("deploy is not running");
        }else{
            Collections.sort(deployEvents, new Comparator<DeployEvent>() {
                @Override
                public int compare(DeployEvent o1, DeployEvent o2) {
                    return ((Long) o2.getStartTime()).compareTo(o1.getStartTime());
                }
            });
            return ResultStat.OK.wrap(deployEvents.get(0));
        }
    }

    @Override
    public List<DeployEvent> listDeployEvent(int deployId) throws IOException {
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deploy in database");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such cluster in database, clusterId = " + deployment.getClusterId());
        }
        if (deployId < 1) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_NOT_LEGAL, "deployId is illegal");
        }
        List<DeployEvent> events = new LinkedList<>();
        List<DeployEvent> deployEvents = deployEventBiz.getEventByDeployId(deployId);
        if (deployEvents != null) {
            events.addAll(deployEvents);
        }

        List<Event> eventList = k8SEventBiz.getEventsByDeployId(deployment.getClusterId(), deployment.getId());
        List<EventInfo> eventInfos = k8SEventBiz.translateEvent(eventList);
        List<DeployEvent> transEvents = translateK8sEvents(deployId, eventInfos);
        if (transEvents != null) {
            events.addAll(transEvents);
        }

        // sort by startTime
        if (events.size() != 0) {
            Collections.sort(events, new Comparator<DeployEvent>() {
                @Override
                public int compare(DeployEvent o1, DeployEvent o2) {
                    return ((Long) o2.getStartTime()).compareTo(o1.getStartTime());
                }
            });
        }

        return events;
    }

    @Override
    public HttpResponseTemp<?> abortDeployOperation(int deployId)
            throws IOException, DeploymentEventException, DaoException, DeploymentTerminatedException {
        checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such deployment:" + deployId);
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
            throw ApiException.wrapResultStat(ResultStat.CLUSTER_WATCHER_NOT_READY);
        }

        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
            throw ApiException.wrapMessage(ResultStat.CLUSTER_NOT_EXIST, " There is no RuntimeDriver for cluster(" + cluster.toString() + ").");
        }
        driver.abortDeployOperation(deployment, CurrentThreadInfo.getUser());
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.ABORT,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap(null);
    }
    
    @Override
    public HttpResponseTemp<?> sreModifyInnerService(int deployId,String lbname,LoadBalancerForDeployDraft loadBalancerDraft,int uid) throws Exception{
    	if(StringUtils.isEmpty(lbname)) {
    		return ResultStat.PARAM_ERROR.wrap("lbname is null");
    	}
    	if (loadBalancerDraft == null) {
    		return ResultStat.PARAM_ERROR.wrap(ResultStat.LOADBALANCER_NOT_LEGAL,"loadBalancerDraft is null");
        }
    	String errInfo = loadBalancerDraft.checkLegality();
    	if (!StringUtils.isBlank(errInfo)) {
            return ResultStat.PARAM_ERROR.wrap(ResultStat.LOADBALANCER_NOT_LEGAL, errInfo);
        }
    	Deployment oldDeploy = deploymentBiz.getDeployment(deployId);
    	if (oldDeploy == null) {
            return ResultStat.PARAM_ERROR.wrap(ResultStat.DEPLOYMENT_NOT_EXIST);
        }
    	if (oldDeploy.getNetworkMode() != NetworkMode.DEFAULT) {
            return ResultStat.PARAM_ERROR.wrap(ResultStat.DEPLOYMENT_UPDATE_FAILED, "can't change netWorkMode ");
        }
    	LoadBalancer oldlb = loadBalancerBiz.getInnerLoadBalancerByDeployId(deployId);
    	LoadBalancerWrapper wrapper = new LoadBalancerWrapper().init(oldDeploy.getClusterId(), oldDeploy.getNamespace());
        //update loadBalancer
        LoadBalancer newlb = loadBalancerDraft.toLoadBalancer(oldDeploy);
        newlb.setName(lbname);
        wrapper.updateLoadBalanceService(newlb);
        if (oldlb == null) {
            loadBalancerBiz.createLoadBalancer(newlb);
            loadBalancerBiz.createLinkDeploy(new DeployLoadBalancerMap(deployId, newlb.getId(), System.currentTimeMillis()));;
        } else {
            newlb.setId(oldlb.getId());
            loadBalancerBiz.updateLoadBalancer(newlb);
        }
        
        //inner_lb 增加到lbc
        LoadBalancerCollection lbcDraft = createlbc(uid);
        //lbc到lb映射
        if (collectionBiz.getResourceByResourceIdAndResourceType(newlb.getId(), resourceType) == null){
        	CollectionResourceMap resourceMap = new CollectionResourceMap(newlb.getId(), 
            		uid, 
                    resourceType, 
                    lbcDraft.getId(),
                    System.currentTimeMillis());
            collectionBiz.addResource(resourceMap);
        }
        
      //更新svc
//        KubeUtils kubeUtils;
//        DeployResourceHandler deployResourceHandler;
//        try {
//            kubeUtils = Fabric8KubeUtils.buildKubeUtils(clusterBiz.getClusterById(newlb.getClusterId()), oldDeploy.getNamespace());
//            deployResourceHandler = getDeployResourceHandler(oldDeploy, kubeUtils);
//        } catch (K8sDriverException e) {
//            throw new DeploymentEventException(e);
//        }
//        io.fabric8.kubernetes.api.model.Service service = new K8sServiceBuilder(newlb).build();
//        kubeUtils.patchService(GlobalConstant.RC_NAME_PREFIX + newlb.getName(), service);
        
        
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.MODIFY,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
    	return ResultStat.OK.wrap("ok");
    }
    
    private DeployResourceHandler getDeployResourceHandler(Deployment deployment, KubeUtils kubeUtils) throws K8sDriverException {
        String deployClass = deployment.getDeploymentType().getDeployClassName();
        if (deployClass == null) {
            throw new K8sDriverException("A deployment must have deployment type");
        }
        Server server = globalBiz.getServer();
        if (server == null) {
            throw new K8sDriverException("Global configuration of Server not set!");
        }
        DeployResourceHandler deployResourceHandler = ReflectFactory.createDeployResourceHandler(deployClass, kubeUtils, deployment, server.getUrl());
        if (deployResourceHandler == null) {
            throw new K8sDriverException("Cannot create deploy handler with deployment :" + deployment);
        }
        return deployResourceHandler;
    }
    
    @Override
    public void modifyInnerService(int deployId, LoadBalancerForDeployDraft loadBalancerDraft) throws Exception {
        checkDeployPermit(deployId, OperationType.MODIFY);
        if (loadBalancerDraft == null) {
            throw ApiException.wrapMessage(ResultStat.LOADBALANCER_NOT_LEGAL, "loadBalancerDraft is null");
        }

        String errInfo = loadBalancerDraft.checkLegality();
        if (!StringUtils.isBlank(errInfo)) {
            throw ApiException.wrapMessage(ResultStat.LOADBALANCER_NOT_LEGAL, errInfo);
        }
        Deployment oldDeploy = deploymentBiz.getDeployment(deployId);
        if (oldDeploy == null) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_NOT_EXIST);
        }
        if (oldDeploy.getNetworkMode() != NetworkMode.DEFAULT) {
            throw ApiException.wrapMessage(ResultStat.DEPLOYMENT_UPDATE_FAILED, "can't change netWorkMode ");
        }
        LoadBalancer oldlb = loadBalancerBiz.getInnerLoadBalancerByDeployId(deployId);
        LoadBalancerWrapper wrapper = new LoadBalancerWrapper().init(oldDeploy.getClusterId(), oldDeploy.getNamespace());
        //update loadBalancer
        LoadBalancer newlb = loadBalancerDraft.toLoadBalancer(oldDeploy);
        wrapper.updateLoadBalanceService(newlb);
        if (oldlb == null) {
            loadBalancerBiz.createLoadBalancer(newlb);
            loadBalancerBiz.createLinkDeploy(new DeployLoadBalancerMap(deployId, newlb.getId(), System.currentTimeMillis()));;
        } else {
            newlb.setId(oldlb.getId());
            loadBalancerBiz.updateLoadBalancer(newlb);
        }
        
        //inner_lb 增加到lbc
        LoadBalancerCollection lbcDraft = createlbc(CurrentThreadInfo.getUserId());
        //lbc到lb映射

        if (collectionBiz.getResourceByResourceIdAndResourceType(newlb.getId(), resourceType) == null){
        	CollectionResourceMap resourceMap = new CollectionResourceMap(newlb.getId(), 
        			CurrentThreadInfo.getUserId(), 
                    resourceType, 
                    lbcDraft.getId(),
                    System.currentTimeMillis());
            collectionBiz.addResource(resourceMap);
        }
        
//        //更新svc
//        KubeUtils kubeUtils;
//        DeployResourceHandler deployResourceHandler;
//        try {
//            kubeUtils = Fabric8KubeUtils.buildKubeUtils(clusterBiz.getClusterById(newlb.getClusterId()), oldDeploy.getNamespace());
//            deployResourceHandler = getDeployResourceHandler(oldDeploy, kubeUtils);
//        } catch (K8sDriverException e) {
//            throw new DeploymentEventException(e);
//        }
//        //注释了svc添加唯一label
////        CollectionResourceMap crm = collectionBiz.getResourceByResourceIdAndResourceType(oldDeploy.getId(), lbcresourceType.DEPLOY);
//        io.fabric8.kubernetes.api.model.Service service = new K8sServiceBuilder(newlb).build();
////        service.getSpec().getSelector().put("unique", getunique(crm.getCollectionId(), oldDeploy));
//        kubeUtils.patchService(GlobalConstant.RC_NAME_PREFIX + newlb.getName(), service);
        
        
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.MODIFY,
                CurrentThreadInfo.getUserId(),
                CurrentThreadInfo.getUserName(),
                "OK",
                "",
                System.currentTimeMillis()
        ));

    }
    
    @Override
    public HttpResponseTemp<?> sreListLoadBalancer(int deployId) throws Exception {
    	Deployment oldDeploy = deploymentBiz.getDeployment(deployId);
        if (oldDeploy == null) {
            return ResultStat.PARAM_ERROR.wrap(ResultStat.DEPLOYMENT_NOT_EXIST);
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, oldDeploy.getClusterId(), Cluster.class);
        if (cluster == null) {
            return ResultStat.PARAM_ERROR.wrap(ResultStat.SERVER_INTERNAL_ERROR, "ClusterId: " + oldDeploy.getClusterId() + " not exists");
        }
        List<LoadBalancer> lbs = loadBalancerBiz.getLoadBalancersByDeploy(deployId);
        if (lbs == null) {
            return ResultStat.OK.wrap(new ArrayList<>(1));
        }
        LoadBalancerWrapper wrapper = null;
        try {
        	wrapper = new LoadBalancerWrapper().init(cluster.getId(), oldDeploy.getNamespace());
        }catch(Exception e) {
        	//nothing to do
        }
        List<LoadBalancerForDeploy> lbForDeploys = new ArrayList<LoadBalancerForDeploy>(lbs.size());
        for (LoadBalancer lb : lbs) {
        	LoadBalancerForDeploy lbForDeploy = new LoadBalancerForDeploy(lb);
        	String ClusterIP = null;
            try {
            	ClusterIP = wrapper.getLoadBalancerService(lb).getSpec().getClusterIP();
            }catch(Exception e) {
            	//nothing to do
            }
            lbForDeploy.setClusterIP(ClusterIP);
            lbForDeploys.add(lbForDeploy);
        }
        Collections.sort(lbForDeploys, new LoadBalancerForDeploy.LoadBalancerForDeployComparator());
        
    	return ResultStat.OK.wrap(lbForDeploys);
    }
    
    @Override
    public List<LoadBalancerForDeploy> listLoadBalancer(int deployId) throws Exception {
        checkDeployPermit(deployId, OperationType.GET);
        Deployment oldDeploy = deploymentBiz.getDeployment(deployId);
        if (oldDeploy == null) {
            throw ApiException.wrapResultStat(ResultStat.DEPLOYMENT_NOT_EXIST);
        }
        List<LoadBalancer> lbs = loadBalancerBiz.getLoadBalancersByDeploy(deployId);
        if (lbs == null) {
            return new ArrayList<>(1);
        }
        List<LoadBalancerForDeploy> lbForDeploys = new ArrayList<LoadBalancerForDeploy>(lbs.size());
        for (LoadBalancer lb : lbs) {
            lbForDeploys.add(new LoadBalancerForDeploy(lb));
        }
        
        Collections.sort(lbForDeploys, new LoadBalancerForDeploy.LoadBalancerForDeployComparator());
        return lbForDeploys;
    }
    
    private List<DeployEvent> translateK8sEvents(int deployId, List<EventInfo> eventInfos) {
        List<DeployEvent> deployEvents = new LinkedList<>();
        int i = 0;
        for (EventInfo eventInfo : eventInfos) {
            deployEvents.add(translateK8sEvent(deployId, eventInfo));
            i++;
            // at most 20 events
            if (i >= 20) {
                break;
            }
        }
        return deployEvents;
    }

    private DeployEvent translateK8sEvent(int deployId, EventInfo eventInfo) {
        DeployEvent deployEvent = new DeployEvent();
        deployEvent.setDeployId(deployId);
        deployEvent.setLastModify(eventInfo.getLastTS());
        String message = String.format("reason:%s, count:%d, message:%s", eventInfo.getReason(), eventInfo.getCount(), eventInfo.getMessage());
        deployEvent.setMessage(message);
        deployEvent.setStartTime(eventInfo.getLastTS());
        deployEvent.setOperation(DeployOperation.KUBERNETES);
        deployEvent.setUserName("system");
        return deployEvent;
    }

    private List<EnvDraft> buildExtraEnv(Cluster cluster) {
        List<EnvDraft> extraEnvs = new LinkedList<>();
        GlobalInfo info = globalMapper.getGlobalInfoByType(GlobalType.SERVER);
        if (info == null) {
            throw ApiException.wrapMessage(ResultStat.PARAM_ERROR, "domeos api is null!");
        }
        extraEnvs.add(new EnvDraft("GDOCKER_SERVER_ADDR", CommonUtil.fullUrl(info.getValue())));
        extraEnvs.add(new EnvDraft("CLUSTER_NAME", cluster.getName()));
        return extraEnvs;
    }

    public VersionString getRCStr(DeploymentDraft deploymentDraft) {
        DeployResourceHandler deployResourceHandler = ReflectFactory.createDeployResourceHandler(
                deploymentDraft.getDeploymentType().getDeployClassName(), null, null);
        if (deployResourceHandler != null) {
            return deployResourceHandler.getVersionString(deploymentDraft);
        } else {
            return null;
        }
    }

    private int getNodeCounts(Deployment deployment, List<LabelSelector> labelSelectors) throws DeploymentEventException {
        if (labelSelectors != null) {
            Map<String, String> labels = new HashMap<>();
            for (LabelSelector labelSelector : labelSelectors) {
                labels.put(labelSelector.getName(), labelSelector.getContent());
            }
            NodeWrapper nodeWrapper;
            try {
                nodeWrapper = new NodeWrapper().init(deployment.getClusterId(), deployment.getNamespace());
            } catch (K8sDriverException e) {
                throw new DeploymentEventException(e);
            }
            List<NodeInfo> nodeInfos = nodeWrapper.getNodeListByLabel(labels);
            if (nodeInfos != null && nodeInfos.size() != 0) {
                return nodeInfos.size();
            } else {
                throw new DeploymentEventException("node selectors select no node");
            }
        }
        return 0;
    }

    private int compareLabelSelectors(Deployment deployment, List<LabelSelector> labelSelectors1, List<LabelSelector> labelSelectors2)
            throws DeploymentEventException {
        Map<String, String> labels1 = new HashMap<>();
        if (labelSelectors1 != null) {
            for (LabelSelector labelSelector : labelSelectors1) {
                labels1.put(labelSelector.getName(), labelSelector.getContent());
            }
        }
        Map<String, String> labels2 = new HashMap<>();
        if (labelSelectors2 != null) {
            for (LabelSelector labelSelector : labelSelectors2) {
                labels2.put(labelSelector.getName(), labelSelector.getContent());
            }
        }
        NodeWrapper nodeWrapper;
        try {
            nodeWrapper = new NodeWrapper().init(deployment.getClusterId(), deployment.getNamespace());
        } catch (K8sDriverException e) {
            throw new DeploymentEventException(e);
        }
        List<NodeInfo> nodeInfos1 = nodeWrapper.getNodeListByLabel(labels1);
        List<NodeInfo> nodeInfos2 = nodeWrapper.getNodeListByLabel(labels2);
        if (nodeInfos1.size() > nodeInfos2.size()) {
            return 1;
        } else if (nodeInfos1.size() < nodeInfos2.size()) {
            return -1;
        } else if (nodeInfos1.equals(nodeInfos2)) {
            return 0;
        } else {
            return 1;
        }
    }

    private boolean isClusterWatcherOK(int clusterId) {
        ClusterWatcherDeployMap watcherDeployMap = clusterBiz.getWacherDeployMapByClusterId(clusterId);
        if (watcherDeployMap == null) {
            return false;
        }
        Deployment deployment = deploymentBiz.getDeployment(watcherDeployMap.getDeployId());
        if (!deployment.deployTerminated()) {
            try {
                DeployEvent event = deployEventBiz.getNewestEventByDeployId(watcherDeployMap.getDeployId());
                EventChecker eventChecker = new EventChecker(deployment, event);
                eventChecker.checkEvent();
                deployment = deploymentBiz.getDeployment(watcherDeployMap.getDeployId());
            } catch (IOException e) {
                logger.warn("get newest event by deploy id error, deployId=" + watcherDeployMap.getDeployId() + ", message: " + e.getMessage());
            } catch (DataBaseContentException e) {
                logger.warn("event or deploy is null, deployId=" + watcherDeployMap.getDeployId());
            }
        }
        return DeploymentStatus.RUNNING.name().equals(deployment.getState());
    }

	@Override
	public HttpResponseTemp<?> sreCheckDeployment(DeploymentDraft deploymentDraft)
			throws Exception {
		if (deploymentDraft == null) {
			return ResultStat.PARAM_ERROR.wrap("deployment is null");
        }
		if(deploymentDraft.getUserId() == 0 || deploymentDraft.getUserId() < 0){
			return ResultStat.PARAM_ERROR.wrap("deployment userId is null!");
		}
        deploymentDraft.setCreatorId(deploymentDraft.getUserId());
        DeployCollection deployCollection = deployCollectionBiz.getDeployCollection(deploymentDraft.getCollectionId());
        if (deployCollection == null) {
        	return ResultStat.PARAM_ERROR.wrap("deploy collection " + deploymentDraft.getCollectionId() + " not exist");
        }
//        //TODO 待修改
//        Cluster cluster = clusterBiz.getClusterByName("uat-k8s-cluster");
//        if (cluster == null) {
//        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
//        }
        //Cluster cluster = clusterBiz.getClusterById(deploymentDraft.getClusterId());
        Cluster cluster = clusterBiz.getClusterByName(deploymentDraft.getClustername());
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
        }

        String deployName = deploymentDraft.getDeployName();
        List<CollectionResourceMap> collectionResourceMaps = collectionBiz.getResourcesByCollectionIdAndResourceType(deploymentDraft.getCollectionId(), ResourceType.DEPLOY);
        if(collectionResourceMaps != null && collectionResourceMaps.size() != 0){
        	for(CollectionResourceMap coMap : collectionResourceMaps){
        		Deployment deployment = deploymentBiz.getById(GlobalConstant.DEPLOY_TABLE_NAME,coMap.getResourceId(), Deployment.class);
        		if(deployment != null){
        			if (deployment.getClusterId() == cluster.getId() &&
        					deployment.getNamespace().equals(deploymentDraft.getNamespace())) {
        				if(deployment.getName().equals(deployName)){
        					return ResultStat.PARAM_ERROR.wrap(deployment.getId());
            			}
        			}
        		}
        	}
        }
        logger.info("sre check deploy succeed, deployName={}, creatorId={}, collectionId={}",
        		deployName, deploymentDraft.getCreatorId(), deploymentDraft.getCollectionId());
        return ResultStat.OK.wrap(null);
	}

	@Override
	public HttpResponseTemp<?> sreCreateDeployment(
			DeploymentDraft deploymentDraft) throws Exception {
		if (deploymentDraft == null) {
			return ResultStat.PARAM_ERROR.wrap("deployment is null");
        }
		if(deploymentDraft.getUserId() == 0 || deploymentDraft.getUserId() < 0){
			return ResultStat.PARAM_ERROR.wrap("deployment userId is null!");
		}
        deploymentDraft.setCreatorId(deploymentDraft.getUserId());
        DeployCollection deployCollection = deployCollectionBiz.getDeployCollection(deploymentDraft.getCollectionId());
        if (deployCollection == null) {
        	return ResultStat.PARAM_ERROR.wrap("deploy collection " + deploymentDraft.getCollectionId() + " not exist");
        }
        if (StringUtils.isEmpty(deploymentDraft.getClustername())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster is null");
        }
        Cluster cluster = clusterBiz.getClusterByName(deploymentDraft.getClustername());
//        Cluster cluster = clusterBiz.getClusterById(deploymentDraft.getClusterId());
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
        }

        String deployName = deploymentDraft.getDeployName();
        List<Deployment> list = deploymentBiz.getListByName(GlobalConstant.DEPLOY_TABLE_NAME, deployName, Deployment.class);
        if (list != null && list.size() != 0) {
            for (Deployment one : list) {
                if (one.getClusterId() == cluster.getId() &&
                        one.getNamespace().equals(deploymentDraft.getNamespace())) {
                	return ResultStat.PARAM_ERROR.wrap("deployment name is exist!");
                }
            }
        }
        deploymentDraft.setClusterId(cluster.getId());
        deploymentDraft.setCreateTime(System.currentTimeMillis());
        //NFS filter
        deploymentDraft = nfsFilter(deploymentDraft, deploymentDraft.getUserId());
        
        Deployment deployment = deploymentDraft.toDeployment();
        deployment.setState(DeploymentStatus.STOP.name());
        deploymentBiz.createDeployment(deployment);
        
        //loadBalancer
        //获取lbc
        LoadBalancerCollection lbcDraft = createlbc(deploymentDraft.getUserId());
        LoadBalancer lb = null;
        LoadBalancerForDeployDraft loadBalancerDraft = deploymentDraft.getLoadBalancerDraft();
        if (deploymentDraft.getNetworkMode() != NetworkMode.HOST && loadBalancerDraft != null) {
            lb = loadBalancerDraft.toLoadBalancer(deployment);
            loadBalancerBiz.createLoadBalancer(lb);
            loadBalancerBiz.createLinkDeploy(new DeployLoadBalancerMap(deployment.getId(), lb.getId(), System.currentTimeMillis()));
            //lbc到lb映射
        	CollectionResourceMap resourceMap = new CollectionResourceMap(lb.getId(), 
        			deploymentDraft.getUserId(), 
                    resourceType, 
                    lbcDraft.getId(),
                    System.currentTimeMillis());
            collectionBiz.addResource(resourceMap);
        }
        
        
        //NFS filter
        deploymentDraft = nfsFilter(deploymentDraft, deploymentDraft.getUserId());
        
        Version version = deploymentDraft.toVersion();
        version.setDeployId(deployment.getId());
        version.setCreateTime(deployment.getCreateTime());
        
        //获取lbc  name,并通过treeid、deploylc、deploymentname拼凑唯一label
        addlabeltoversion(version,deployCollection.getId());
        
        //增加超分超卖配置
        double cpupercent = globalBiz.getCpuPercent();
        double memorypercent = globalBiz.getMemoryPercent();
        for(ContainerDraft cd:version.getContainerDrafts()) {
        	cd.setCpuRequest(cd.getCpuRequest() * cpupercent);
        	cd.setMemRequest(cd.getMemRequest() * memorypercent);
        }
        
        try {
            versionBiz.insertVersionWithLogCollect(version, cluster);
        } catch (Exception e) {
            // failed
            deploymentBiz.removeById(GlobalConstant.DEPLOY_TABLE_NAME, deployment.getId());
            versionBiz.removeById(GlobalConstant.VERSION_TABLE_NAME, version.getId());
            return ResultStat.PARAM_ERROR.wrap(e.getMessage());
        }
        
	    // add volume_deploy_map
        for(ContainerConsole cc : deploymentDraft.getContainerConsoles()) {
        	for(VolumeMountConsole vmc: cc.getVolumeMountConsoles()) {
        		if ("PERSISTENTVOLUMECLAIM".equals(vmc.getVolumeType().name())) {
        			String storageName = vmc.getVolumePVC().getClaimName().split("-pvc")[0];
        			Storage storage = storageBiz.getStorageByName(storageName);
        			VolumeDeployMap volumeDeployMap = new VolumeDeployMap();
        			volumeDeployMap.setDeployId(deployment.getId());
        			volumeDeployMap.setVersionId(version.getId());
        			volumeDeployMap.setVolumeId(storage.getId());
        			volumeDeployMap.setCreateTime(System.currentTimeMillis());
        			storageBiz.addVolumeVersionMount(volumeDeployMap);
        		}
        	}
        }
        
        User user =  userService.getUser(deploymentDraft.getUserId()); 

        CollectionResourceMap resourceMap = new CollectionResourceMap(deployment.getId(),
        		user.getId(),
                ResourceType.DEPLOY,
                deploymentDraft.getCollectionId(),
                System.currentTimeMillis());
        collectionBiz.addResource(resourceMap);
        
        
        //获取最新删除的deploy并绑定到之前的service
        //获取唯一label
        DeployCollection deployc = deployCollectionBiz.getDeployCollectionByID(deploymentDraft.getCollectionId());
        String unique =  deployc.getName() + "-" + deployment.getName() + "-" + String.valueOf(deployc.getTreeId());
        List<Deployment> olddeploys = deploymentBiz.getRemoveDeployment(cluster.getId(), deployName);
        //匹配新旧deploy
        Deployment olddeploy = null;
        if (olddeploys.size()!=0) {
        	for(Deployment oldd:olddeploys) {
        		int b=0;
        		for (CustomLabel ctl:version.getCustomLabels()) {
        			if (ctl.getKey().equals("unique")) {
        				if (ctl.getValue().equals(unique)) {
        					olddeploy = oldd;
        					b=1;
        					break;
        				}
        			}
        		}
        		if (b==1) {
        			break;
        		}
        	}	
        }
        
        if (olddeploy != null) {
        	//更新lb表
        	LoadBalancer oldlb = loadBalancerBiz.getInnerLoadBalancerByDeployId(olddeploy.getId());
        	if (oldlb != null) {
        		KubeServiceDraft kbsd = oldlb.getServiceDraft();
            	if (kbsd != null) {
            		kbsd.setDeployId(deployment.getId());
            		kbsd.setDeployName(deployment.getName());
            		loadBalancerBiz.updateLoadBalancer(oldlb);
            		//更新lb_deploy_map表
                	DeployLoadBalancerMap dlbmap = new DeployLoadBalancerMap();
                	dlbmap.setLoadBalancerId(oldlb.getId());
                	dlbmap.setDeployId(deployment.getId());
                	loadBalancerBiz.updateLinkDeploy(dlbmap);
                	
                	//更新实际svc
                	LoadBalancerWrapper wrapper = new LoadBalancerWrapper().init(deployment.getClusterId(), deployment.getNamespace());
                	wrapper.updateLoadBalanceService(oldlb);
            	}
        	}
        }
        
        
        operationHistory.insertRecord(new OperationRecord(
                resourceMap.getResourceId(),
                resourceMap.getResourceType(),
                OperationType.SET,
                user.getId(),
                user.getUsername(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        logger.info("sre create deploy succeed, deployId={}, creatorId={}, collectionId={}",
                deployment.getId(), deploymentDraft.getCreatorId(), deploymentDraft.getCollectionId());

        return ResultStat.OK.wrap(deployment.getId());
	}
	
	private DeploymentDraft nfsFilter(DeploymentDraft deploymentDraft, int uid) {
		User user =  userService.getUser(uid);
		for(ContainerConsole containerConsole : deploymentDraft.getContainerConsoles()) {
			if(CollectionUtils.isNotEmpty(containerConsole.getVolumeMountConsoles())){
				for(VolumeMountConsole volumeMountConsole : containerConsole.getVolumeMountConsoles()) {
					if(volumeMountConsole.getVolumeType().equals(VolumeType.NFS)) {
						List<Storage> storages = storageBiz.getStorageByPathAndServer(volumeMountConsole.getNfs().getPath(), volumeMountConsole.getNfs().getServer());
						Storage storage;
						if(storages == null || storages.size() == 0) {
							// create storage
							storage = new Storage();
							storage.setClusterId(deploymentDraft.getClusterId());
							storage.setNamespace(deploymentDraft.getNamespace());
							storage.setName(deploymentDraft.getDeployName() + "-version1");
							storage.setPath(volumeMountConsole.getNfs().getPath());
							storage.setServer(volumeMountConsole.getNfs().getServer());
							storage.setType("nfs");
							storage.setCreateTime(System.currentTimeMillis());
							storage.setCreatorId(uid);
							storageService.sreSetStorage(user, storage);
						} else {
							storage = storages.get(0);
						}
						volumeMountConsole.setVolumeType(VolumeType.PERSISTENTVOLUMECLAIM);
						VolumePVC volumePVC = new VolumePVC();
						volumePVC.setClaimName(storage.getName() + "-pvc");
						volumePVC.setReadOnly(volumeMountConsole.isReadonly());
						volumeMountConsole.setVolumePVC(volumePVC);
					}
				}
			}
		}
		
		//设置收集日志
        Set<String> path = null;
        if(CollectionUtils.isNotEmpty(deploymentDraft.getContainerConsoles())){
        	for(ContainerConsole console : deploymentDraft.getContainerConsoles()){
        		// mount log path
        		path = new HashSet<String>();
        		if(CollectionUtils.isNotEmpty(console.getEnvs())){
        			for(EnvDraft envDraft : console.getEnvs()){
        				if(envDraft.getValue().startsWith("/")){
        					String[] logsPath = StringUtils.split(envDraft.getValue(),",");
        		        	for(int i = 0 ; i < logsPath.length ; i ++){
        		        		path.add(new File(logsPath[i]).getParent());
        		        	}
        				}
        			}
        		}
        		if(CollectionUtils.isNotEmpty(path)){
                	List<VolumeMountConsole> mountConsoles = new ArrayList<VolumeMountConsole>();
                	List<VolumeMountDraft> mountDrafts = new ArrayList<VolumeMountDraft>();
                	Iterator<String> i = path.iterator();
                	int index = 0;
                    while(i.hasNext()){
                    	String p = i.next();
                    	VolumeMountConsole volumeMountConsole = new VolumeMountConsole();
                		volumeMountConsole.setName("log-" + index);
                		volumeMountConsole.setContainerPath(p);
                		volumeMountConsole.setVolumeType(VolumeType.EMPTYDIR);
                		mountConsoles.add(volumeMountConsole);
                    	
                    	VolumeMountDraft volumeMountDraft = new VolumeMountDraft();
                		volumeMountDraft.setMountPath(p);
                		volumeMountDraft.setName("log-" + index);
                		mountDrafts.add(volumeMountDraft);
                		
                		index ++;
                    }  
                	console.setVolumeMountConsoles(mountConsoles);
                	console.setVolumeMountDrafts(mountDrafts);
                }
        	}
            
        }
		return deploymentDraft;
	}
	
	//根据 namespace获取环境
	private String getEnv(String namespaces){
		if(namespaces.indexOf("sit") > 0){
			return "sit";
		}
		if(namespaces.indexOf("uat") > 0){
			return "uat";
		}
		if(namespaces.indexOf("pre") > 0){
			return "pre";
		}
		if(namespaces.indexOf("live") > 0){
			return "live";
		}
		return StringUtils.EMPTY;
	}

	@Override
	public HttpResponseTemp<?> sreStartDeployment(int deployId, int versionId,
			int replicas, int userId) throws IOException, DeploymentEventException,
			DaoException, DeploymentTerminatedException {
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("deployment is null");
        }
        Version version = versionBiz.getVersion(deployId, versionId);
        if (version == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such version:" + versionId);
        }
        if (version.isDeprecate()) {
        	return ResultStat.PARAM_ERROR.wrap("can't start deprecated version");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready!");
        }
        
        //检测namespace是否存在，不存在就创建
        try {
			NodeWrapper nodeWrapper = new NodeWrapper().init(deployment.getClusterId(), null);
			List<NamespaceInfo> namespaceinfos = nodeWrapper.getAllNamespaces();
			boolean isnsexist = false;
			for (NamespaceInfo namespaceinfo:namespaceinfos) {
				if (namespaceinfo.getName().equals(deployment.getNamespace())) {
					isnsexist = true;
				}
			}
			if (!isnsexist) {
				List<String> namespaces = new ArrayList<>();
				namespaces.add(deployment.getNamespace());
				if (!nodeWrapper.setNamespaces(namespaces)) {
	                // todo: add operation history
	                return ResultStat.PARAM_ERROR.wrap("create namespaces false");
	            }
			}
        } catch (K8sDriverException e1) {
			return ResultStat.PARAM_ERROR.wrap("connect k8s error");
		}

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.DEPLOYING);
        deployment.setState(DeploymentStatus.DEPLOYING.name());
        deployment.setLastUpdateTime(System.currentTimeMillis());
        if (replicas > 0) {
            deployment.setDefaultReplicas(replicas);
        }

        deploymentBiz.update(deployment);
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
        }
        
        User user =  userService.getUser(userId); 
        
        //version添加默认label
        

        try {
            List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
            driver.startDeploy(deployment, version, user, allExtraEnvs);
            operationHistory.insertRecord(new OperationRecord(
                    deployId,
                    ResourceType.DEPLOY,
                    OperationType.START,
                    user.getId(),
                    user.getUsername(),
                    "OK",
                    "",
                    System.currentTimeMillis()
            ));
        } catch (DriverException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            logger.error("deployment start failed!",e.getMessage());
            return ResultStat.PARAM_ERROR.wrap("deployment start failed!");
        }
        return ResultStat.OK.wrap();
	}

	@Override
	public HttpResponseTemp<?> sreAbortDeployOperation(int deployId, int userId)
			throws IOException, DeploymentEventException, DaoException,
			DeploymentTerminatedException {
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deployment:" + deployId);
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready!");
        }

        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("There is no RuntimeDriver for cluster(" + cluster.toString() + ").");
        }
        
        User user =  userService.getUser(userId); 
        
        driver.abortDeployOperation(deployment, user);
        
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.ABORT,
                user.getId(),
                user.getUsername(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        return ResultStat.OK.wrap();
	}

	@Override
	public HttpResponseTemp<?> sreStopDeployment(int deployId, int userId)
			throws IOException, DeploymentEventException,
			DeploymentTerminatedException {
		Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deployment:" + deployId);
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready!");
        }
        
        DeployEvent deployEvent = deployEventBiz.getNewestEventByDeployId(deployId);
        if (!deployEvent.eventTerminated() && deployEvent.getOperation() != DeployOperation.STOP) {
        	return ResultStat.PARAM_ERROR.wrap("Fail the current event!");
        } else if (deployEvent.getOperation() == DeployOperation.STOP) {
        	return ResultStat.PARAM_ERROR.wrap("You are already in stop status");
        }

        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("There is no RuntimeDriver for cluster(" + cluster.toString() + ").");
        }

        User user =  userService.getUser(userId); 
        
        try {
            deploymentBiz.updateState(GlobalConstant.DEPLOY_TABLE_NAME, DeploymentStatus.STOPPING.name(), deployId);
            driver.stopDeploy(deployment, user);
            // add operation record
            operationHistory.insertRecord(new OperationRecord(
                    deployId,
                    ResourceType.DEPLOY,
                    OperationType.STOP,
                    user.getId(),
                    user.getUsername(),
                    "OK",
                    "",
                    System.currentTimeMillis()
            ));
        } catch (DeploymentEventException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            logger.error("deployment stop failed!",e.getMessage());
            return ResultStat.PARAM_ERROR.wrap("deployment stop failed!");
        }
        return ResultStat.OK.wrap();
	}

	@Override
	public HttpResponseTemp<?> sreStartRollback(int deployId, int versionId,
			int replicas, Policy policy, int userId) throws IOException,
			DeploymentEventException, DaoException,
			DeploymentTerminatedException {
		Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("deployment is null");
        }
        Version version = versionBiz.getVersion(deployId, versionId);
        if (version == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such version:" + versionId);
        }
        if (version.isDeprecate()) {
        	return ResultStat.PARAM_ERROR.wrap("can't start deprecated version");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready!");
        }

        User user =  userService.getUser(userId); 
        
        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.BACKROLLING);
        
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
        }
        try {
            List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
            if (replicas > 0) {
                deployment.setDefaultReplicas(replicas);
            }
            driver.rollbackDeploy(deployment, versionId, allExtraEnvs, user, policy);
        } catch (DeploymentEventException e) {
            deploymentStatusManager.failedEventForDeployment(deployId, null, e.getMessage());
            logger.error("deployment update failed!",e.getMessage());
            return ResultStat.PARAM_ERROR.wrap("deployment update failed!");
        }

        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.BACKROLLING.name());
        deploymentBiz.update(deployment);
        
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.ROLLBACK,
                user.getId(),
                user.getUsername(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap();
	}

	@Override
	public HttpResponseTemp<?> sreScaleUpDeployment(int deployId,
			int versionId, int replicas,int userId) throws IOException,
			DeploymentEventException, DaoException,
			DeploymentTerminatedException {
//		checkDeployPermit(deployId, OperationType.MODIFY);
        Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
        	return ResultStat.PARAM_ERROR.wrap("watcher cannot be scale");
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready");
        }

        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.UPSCALING);
        
        deployment.setDefaultReplicas(replicas);
        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.UPSCALING.name());
        deploymentBiz.update(deployment);

        User user =  userService.getUser(userId); 
        
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
        }
        List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
        driver.scaleUpDeployment(deployment, versionId, replicas, allExtraEnvs, user);
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.SCALEUP,
                user.getId(),
                user.getUsername(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap();
	}

	@Override
	public HttpResponseTemp<?> sreScaleDownDeployment(int deployId,
			int versionId, int replicas, int userId) throws IOException,
			DeploymentEventException, DaoException,
			DeploymentTerminatedException {
		Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
        	return ResultStat.PARAM_ERROR.wrap("watcher cannot be scale");
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready");
        }


        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.DOWNSCALING);

        deployment.setDefaultReplicas(replicas);
        deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.DOWNSCALING.name());
        deploymentBiz.update(deployment);

        User user =  userService.getUser(userId); 
        
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        if (driver == null) {
        	return ResultStat.PARAM_ERROR.wrap("cluster not exist!");
        }
        List<EnvDraft> allExtraEnvs = buildExtraEnv(cluster);
        driver.scaleDownDeployment(deployment, versionId, replicas, allExtraEnvs, user);
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.SCALEDOWN,
                user.getId(),
                user.getUsername(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        
        return ResultStat.OK.wrap();
	}
	
	@Override
	public HttpResponseTemp<?> sreRestartDeployment(int deployId,int period,double percent,int userid) throws DeploymentEventException{
		Deployment deployment = deploymentBiz.getDeployment(deployId);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such deployment:" + deployId);
        }

        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("no such clusterId: " + deployment.getClusterId());
        }

        if (VersionType.WATCHER.equals(deployment.getVersionType())) {
        	return ResultStat.PARAM_ERROR.wrap("watcher cannot be scale");
        }
        if (!VersionType.WATCHER.equals(deployment.getVersionType()) && !isClusterWatcherOK(cluster.getId())) {
        	return ResultStat.PARAM_ERROR.wrap("cluster watcher not ready");
        }
        deploymentStatusManager.checkStateAvailable(DeploymentStatus.valueOf(deployment.getState()), DeploymentStatus.UPDATING);
        
        
        RuntimeDriver driver = ClusterRuntimeDriver.getClusterDriver(cluster.getId());
        
        //注册更新事件
        User user =  userService.getUser(userid); 
        List<Version> versions = driver.getCurrnetVersionsByDeployment(deployment);
        try {
			driver.reStartUpdate(deployment, versions.get(0).getVersion(), user);
		} catch (IOException | DeploymentEventException | DeploymentTerminatedException e1) {
			// TODO Auto-generated catch block
			return ResultStat.PARAM_ERROR.wrap("重启失败");
		}
        
        //异步重启
		executorService.execute(new RestartPod(cluster, deployment, period, percent));
		
		//更新deploy的状态
		deployment.setLastUpdateTime(System.currentTimeMillis());
        deployment.setState(DeploymentStatus.UPDATING.name());
        try {
			deploymentBiz.update(deployment);
		} catch (DaoException e) {
			// TODO Auto-generated catch block
			return ResultStat.PARAM_ERROR.wrap("error");
		}
		return ResultStat.OK.wrap("ok");
	}
	
	private class RestartPod implements Runnable{
		
		private Cluster cluster;
		
		private Deployment deployment;
		
		private int period;
		
		private double percent;
		
		public RestartPod(Cluster cluster,Deployment deployment,int period,double percent){
			this.cluster = cluster;
			this.deployment = deployment;
			this.period = period;
			this.percent = percent;
		}

		@Override
		public void run() {
			Map<String,String> selectors = new HashMap<>();
			selectors.put(GlobalConstant.DEPLOY_ID_STR, String.valueOf(deployment.getId()));
			try {
				KubeUtils client = Fabric8KubeUtils.buildKubeUtils(cluster, deployment.getNamespace());
				PodList podlist = client.listPod(selectors);
				int count = 0;
				int per_restart_count = (int)(podlist.getItems().size() * percent);
				for(Pod pod:podlist.getItems()) {
					if(count == per_restart_count) {
						try {
							Thread.sleep(period);
							count = 0;
						}catch (InterruptedException e) {
							logger.error("restart is error!" + e);
						}
					}
					client.deletePod(pod.getMetadata().getName());
					count += 1;
				}
			}catch (K8sDriverException e) {
				logger.error("connection k8s cluster is error!" + e);
			}
		}
		
	}
	

	@Override
	public HttpResponseTemp<?> sreStartUpdate(int deployId, int versionId,
			int replicas, Policy policy, int userId) throws IOException,
			DeploymentEventException, DaoException,
			DeploymentTerminatedException {
		
		AutoUpdateInfo autoUpdateInfo = new AutoUpdateInfo();
        autoUpdateInfo.setDeployId(deployId);
        autoUpdateInfo.setVersionId(versionId);
        autoUpdateInfo.setReplicas(replicas);
        autoUpdateInfo.setPolicy(policy);

        User user =  userService.getUser(userId); 
        
        HttpResponseTemp<?> httpResponseTemp = sreStartUpdate(autoUpdateInfo, true, user);
        // add operation record
        operationHistory.insertRecord(new OperationRecord(
                deployId,
                ResourceType.DEPLOY,
                OperationType.UPDATE,
                user.getId(),
                user.getUsername(),
                "OK",
                "",
                System.currentTimeMillis()
        ));
        return httpResponseTemp;
	}

	@Override
	public HttpResponseTemp<?> sreListPodsByDeployId(int deployId, int userId)
			throws IOException, DeploymentEventException, DaoException,
			DeploymentTerminatedException, ParseException, K8sDriverException {
		Deployment deployment = deploymentBiz.getById(GlobalConstant.DEPLOY_TABLE_NAME, deployId, Deployment.class);
        if (deployment == null) {
        	return ResultStat.PARAM_ERROR.wrap("The deployment with id=" + deployId + " does not exist.");
        }
        Cluster cluster = clusterBiz.getById(GlobalConstant.CLUSTER_TABLE_NAME, deployment.getClusterId(), Cluster.class);
        if (cluster == null) {
        	return ResultStat.PARAM_ERROR.wrap("The cluster with clusterId " + deployment.getClusterId() + " does not exist.");
        }
        if(DeploymentStatus.valueOf(deployment.getState()) == DeploymentStatus.STOP){
        	return ResultStat.PARAM_ERROR.wrap("The deployment is stop " + deployment.getId() + " status " + deployment.getState());
        }
        Map<String, String> labels = new HashMap<>();
        labels.put(GlobalConstant.DEPLOY_ID_STR, String.valueOf(deployment.getId()));
        NodeWrapper nodeWrapper = new NodeWrapper().init(cluster.getId(), deployment.getNamespace());
        
        PodList podList = nodeWrapper.getPods(labels);
        if (podList == null || podList.getItems() == null || podList.getItems().isEmpty()) {
            return ResultStat.OK.wrap(new ArrayList<>(1));
        }
        List<Instance> instances = new ArrayList<>(podList.getItems().size());
        for (Pod pod : podList.getItems()) {
            Instance instance = new Instance();
            instance.setDeloyId(deployId);
            instance.setDeployName(deployment.getName());
            instance.setNamespace(pod.getMetadata().getName());
            if (pod.getMetadata() != null) {
                instance.setInstanceName(pod.getMetadata().getName());
                if (pod.getMetadata().getLabels() != null && pod.getMetadata().getLabels().containsKey(GlobalConstant.VERSION_STR)) {
                    instance.setVersionId(Integer.valueOf(pod.getMetadata().getLabels().get(GlobalConstant.VERSION_STR)));
                }
            }
            if (pod.getSpec() != null) {
                instance.setHostName(pod.getSpec().getNodeName());
            }
            if (pod.getStatus() != null) {
                instance.setStartTime(DateUtil.string2timestamp(pod.getStatus().getStartTime(), TimeZone.getTimeZone(GlobalConstant.UTC_TIME)));
                instance.setPodIp(pod.getStatus().getPodIP());
                instance.setHostIp(pod.getStatus().getHostIP());
                if (pod.getStatus().getContainerStatuses() != null) {
                    for (ContainerStatus containerStatus : pod.getStatus().getContainerStatuses()) {
                        if (StringUtils.isBlank(containerStatus.getContainerID())) {
                            continue;
                        }
                        String containerId = containerStatus.getContainerID().split("docker://")[1];
                        instance.addContainer(new Container(containerId, containerStatus.getName(), containerStatus.getImage()));
                    }
                }
            }
            instance.setStatus(PodUtils.getPodStatus(pod));
            instances.add(instance);
        }
        return ResultStat.OK.wrap(instances);
	}

}